/**
 * 采集搜狗123
 * @link https://123.sogou.com/
 */
const fs = require('fs');
const axios = require('axios')
const cheerio = require('cheerio')
// Node 环境当中不支持 GBK 编码，所以需要引用 iconv-lite 模块来转码
const iconv = require('iconv-lite')

axios.defaults.timeout = 10000; // 10秒超时
const SITE_NAME = 'sougou123';
const RUNTIME_PATH = './runtime/' + SITE_NAME + '/';
const SQL_FILE_PATH = RUNTIME_PATH + SITE_NAME + '.sql';
const JSON_FILE_PATH = RUNTIME_PATH + SITE_NAME + '.json';
if (!fs.existsSync(RUNTIME_PATH)) {
    fs.mkdirSync(RUNTIME_PATH)
}


(async function () {
    let links = [
        { name: 'kuzhan', text: '酷站', link: 'https://123.sogou.com/' },
    ];
    let list = [];
    for (let item of links) {
        list = list.concat(await getJson(item.name, item.text, item.link));
    }
    fs.writeFileSync(JSON_FILE_PATH, JSON.stringify(list));
    fs.writeFileSync(SQL_FILE_PATH, json2sql(list).join("\n"))
    console.log('列表采集完毕.');
})()


async function getJson(name, text, url) {
    let tempFile = RUNTIME_PATH + name + ".html", html = '';
    if (!fs.existsSync(tempFile)) {
        html = (await axios.get(url)).data;
        fs.writeFileSync(tempFile, html);
    } else {
        html = fs.readFileSync(tempFile);
    }
    // html=iconv.decode(html,'gbk');
    let $ = cheerio.load(html);
    let data = { text: text, data: [] };
    $('.coolsite li').each(function (i, el) {
        let $el = $(el), $elFirst = $el.children().eq(0);
        let item = { text: $elFirst.text(), data: [] };
        $el.find('span a').each(function (i2, el2) {
            let $el2 = $(el2);
            item.children.push({ text: $el2.text(), link: $el2.attr('href') });
        })
        data.children.push(item);
        // console.log(data);
    })
    return data;
}
function json2sql(json) {
    let sqlArr = [], table = 'tbl_navigate', id = 0, pid = 0;
    json.forEach(item => {
        id++;
        sqlArr.push(`INSERT INTO ${table} (id, pid, text, link) VALUES ('${id}', 0, '${item.text}', '${item.link}');`);
        pid = id;
        item.children.forEach(item2 => {
            id++;
            sqlArr.push(`INSERT INTO ${table} (id, pid, text, link) VALUES ('${id}', ${pid}, '${item2.text}', '${item2.link}');`);
            pid = id;
            item2.children.forEach(item3 => {
                id++;
                sqlArr.push(`INSERT INTO ${table} (id, pid, text, link) VALUES ('${id}', ${pid}, '${item3.text}', '${item3.link}');`);
            })
        })
    })
    return sqlArr;
}